package indexbuilder;

import htmlparser.DoubanHtmlParser;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Date;

import net.paoding.analysis.analyzer.PaodingAnalyzer;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.FSDirectory;
import org.htmlparser.util.ParserException;

import common.Activity;

public class IndexBuilder {
	private IndexBuilder() {}

	static final File INDEX_DIR = new File("index");
	static final File DOUBANROOTDIR = new File("/home/zhangxu/heritrix-1.14.3/jobs/" +
							"myfifth-farewell-20100429080845690/mirror/www.douban.com/event/");
  
	static void indexDocs(IndexWriter writer, File fpath) throws ParserException, CorruptIndexException, IOException{
        File[] children = fpath.listFiles();

        for (File child : children) {
            if (child.isDirectory()) {
            	indexDocs(writer, child);
            } else {
            	String path = child.getAbsolutePath();
                Activity act = new Activity();
                if (DoubanHtmlParser.parserToActivity(act, getDoubanURLFromPath(path), path, "UTF-8")){
					//System.out.println("adding activity " + act.getName());
					try {
						writer.addDocument(ActivityDocument.getDocument(act));
					}
					// at least on windows, some temporary files raise this exception with an "access denied" message
					// checking if the file can be read doesn't help
					catch (FileNotFoundException fnfe) {
						System.out.println("caught FileNotFoundException");
			        }
                }
			}
	    }
	}
	
	public static String getDoubanURLFromPath(String path){
		//....../mirror/www.douban.com/event/10250928/index.html
		int start = path.indexOf("www.douban.com");
		int end = path.indexOf("/index");
		return path.substring(start, end);
	}
	 
	public static void main(String[] args) {
		if (INDEX_DIR.exists()) {
			System.out.println("Cannot save index to '" +INDEX_DIR+ "' directory, please delete it first");
			System.exit(1);
		}
		try {
			Date start = new Date();
			IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR), new PaodingAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
			System.out.println("Indexing to directory '" +INDEX_DIR+ "'...");
			indexDocs(writer, DOUBANROOTDIR);
			System.out.println("Optimizing...");
			writer.optimize();
			writer.close();
			Date end = new Date();
			System.out.println(end.getTime() - start.getTime() + " total milliseconds");
		} catch (ParserException e) {
			e.printStackTrace();
		} catch (CorruptIndexException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
  }
}
